#!/usr/bin/env python
# coding=utf-8

import os, sys, re, traceback, logging
import urllib
import time
import datetime
import getopt
import urllib2
from Route import *

vacation_routenum = re.compile(r'''pages=(\d+)>最后一页''', re.X|re.S)

vacation_routeurl = re.compile(r'''<div\s*class=.censshdva[\S\s]+<ul>\s*<li>[\S\s]+</li>\s*</ul>\s*<div\s*class=.newclassva''', re.X|re.S)

route_title_re = re.compile(r'''<div\s*class=.t_detail_titile[^>]>\s*<h1>\s*([\S\s]+)\s*</h1>''', re.X|re.S)
route_sights_re = re.compile(r'''class=.destination_outer[\S\s]+<ul>\s*(<li>[\S\s]+</li>)\s*</ul>\s*</div>\s*</div>''', re.X|re.S)
route_price_re = re.compile(r'''<div\s*class=.t_detail_info_right_price[^>][\S\s]+[¥]+([\S\s]+)起</b>''', re.X|re.S)
route_detail_re = re.compile(r'''<h2>特色</h2>[\S\s]+?<font\s*color=..618e1d.>([\S\s]+?)</td>''', re.X|re.S)

route_dates_re = re.compile(r'''onclick=.goto_booking[\S\s]+?(\d{4}\-\d{2}\-\d{2})[\S\s]+?\)''', re.X|re.S)

route_planes_re = re.compile(r'''<div\s*class=.t_detail_trip_text[^>]+>\s*([\S\s]+?)\s*</div>''', re.X|re.S)

route_trip_all_re = re.compile(r'''<div\s*class=.t_detail_trip_all2[^>]+>\s*([\S\s]+?)\s*</div>''', re.X|re.S)

re_h=re.compile('<[\S\s]*>') #取出信息中夹杂的html标签

route_dayArray = [('一天','1'),('二天','2'),('三天','3'),('四天','4'),('五天','5'),('六天','6'),('七天','7'),('八天','8'),('九天','9'),('十天','10')]

http_url = "http://www.ctsgz.cn"  #广州中国旅行社
route_telphone = "020-86000577"

def do_crawl():
    try:
        server_url_data = [
                ('广州','跟团游', 'http://www.ctsgz.cn/tpl/index/tuandui_query.do?pages=1', ),
               # ('广州','自由行', 'http://www.ctsgz.cn/tpl/index/tuandui_query.do?pages=1', ),
        ]
        #
        for depcity,routetype,server_url in server_url_data:
            html = urllib2.urlopen(server_url).read()
            if depcity.find('广州')>-1 and routetype.find('跟团游')>-1:
                routenum=''
                html = urllib2.urlopen(server_url).read()
                vacation_route_num = vacation_routenum.search(html)
                if vacation_route_num:
                    u=util.strutil(vacation_route_num.group(1))
                    routenum=u.filter_tags()
                    for i in range(int(routenum)):
                        page_url ="http://www.ctsgz.cn/tpl/index/tuandui_query.do?pages="+str(i+1)
                        vacation_route_url=""
                        html = urllib2.urlopen(page_url).read()
                        vacation_route_url = vacation_routeurl.findall(html)
                        vacation_route_url = re.compile(r'''<a\s*href=.([\S\s]+?)">([\S\s]+?)</a>[\S\s]+?<font>([\S\s]+?)</font></li>''', re.X|re.S).findall(vacation_route_url[0])
                        for url,title,price in vacation_route_url:
                            if price.find("参")>-1:
                                if url:
                                    route_price = re.compile(r'''([0-9]+)''', re.X|re.S).findall(price)
                                    route_title = title.split(".")[1].replace("&nbsp;","")
                                    #print route_price[0]+route_title
                                    getRoutesHtml(depcity,routetype,url,route_price[0],route_title)
            if depcity.find('广州')>-1 and routetype.find('自由行')>-1:
                page_url="http://wwwhttp://www.ctsgz.cn/tpl/index/tuandui_query.do?pages=1.aijiatu.com/chu/product/b2c_auto_search?departure_city_id=1234&text_search=%E4%B8%AD%E6%96%87%2F%E6%8B%BC%E9%9F%B3&new_category_id=&departure[date]=&pricing_order=asc_order&submit=&page="+str(i+1)+"&destination=&calendar[date]=&text="
                
    except:
        print traceback.format_exc(sys.exc_info())



def getRoutesHtml(depcity,routetype,url,route_price,route_title):
    try:
        print "route_url:"+http_url+url
        route_url = http_url+url
        #route_url = "http://www.ctsgz.cn/tpl/index/20070713170103.html"
        #此处开始获取线路信息
        html = urllib2.urlopen(route_url).read()
        #
        route_sights=""
        #
        print route_title
        route_day = ""
        route_days = re.compile(r'''([0-9]+)天''', re.X|re.S).search(route_title)
        #route_days2 = re.compile(r'''(..)天''', re.X|re.S).findall(route_title)
        if route_days:
            u=util.strutil(route_days.group(1))
            route_day=u.filter_tags()
        else:
            if route_title:
                for day1,day2 in route_dayArray:
                    if route_title.find(day1)>-1:
                        route_day = day2
                        break
                    else:
                        route_day = 1
        print route_day
        #
        route_detail = route_detail_re.search(html)
        if route_detail:
           u=util.strutil(route_detail.group(1))
           route_detail=u.filter_tags().strip()
        else:
           route_detail = ""  
        #
        route_dates=""
        route_date = route_dates_re.findall(html)
        for i in route_date:
            route_dates = route_dates+i+";"
        #print route_dates
        #
        route_plane = getPlane(html)
        #print route_plane
        #
        route_hotel = ""
        route_meal = ""
        route_traffic01 = ""
        route_tickets = ""
        route_payown = ""
        route_trip = route_trip_all_re.findall(html)
        if route_trip:
            for route_trips in route_trip[0].split("<br />"):
                if route_trips.find("住宿")>-1:
                    if route_trips.find("：")>-1:
                        route_trips = (route_trips.split("："))[1]
                    route_hotel = re_h.sub('',route_trips)
                if route_trips.find("餐饮")>-1:
                    if route_trips.find("：")>-1:
                        route_trips = (route_trips.split("："))[1]
                    route_meal = re_h.sub('',route_trips)
                if route_trips.find("交通")>-1:
                    if route_trips.find("：")>-1:
                        route_trips = (route_trips.split("："))[1]
                    route_traffic01 = re_h.sub('',route_trips)
                if route_trips.find("门票")>-1:
                    if route_trips.find("：")>-1:
                        route_trips = (route_trips.split("："))[1]
                    route_tickets = re_h.sub('',route_trips)
            route_payown = route_trip[1].strip().replace("<p>","").replace("</p>","")
            #print route_payown
        #
        route_go_back=""
        route_go_back = getTrafficType(route_title)
        #print route_go_back
        #入库
        route = Route()
        route.site = '31'
        route.supplier_url = "http://www.ctsgz.cn"
        route.supplier = "广州中国旅行社"
        route.title = route_title
        route.rout_url = route_url
        route.outcity = depcity
        route.sights = route_sights
        route.telphone = route_telphone
        route.detail = route_detail
        route.dest = route_sights
        route.guide = ""
        route.type = routetype
        route.dates = route_dates
        route.price = int(route_price)
        route.go_t = route.back_t = route_go_back
        route.plane = route_plane
        route.days = route_day

        route.hotel = route_hotel
        route.meal = route_meal
        route.traffic01 = route_traffic01
        route.tickets = route_tickets
        route.payown = route_payown
        
        getschedules = getSchedules(html)
        if getschedules=='no':
            print "==no schedule=="
        else:
            route.schedules = getschedules
            route.updRoute_Schedules()
        print route_price
        route.updRoute()

        #print "++++++"
        #print "++++++"
        #print "++++++"
    except Exception,e:
        print traceback.format_exc(sys.exc_info())
    #print routenum+"dddd"





Schedules_place_re = re.compile(r'''<h2>行程安排</h2>[\S\s]+?<font\s*color=..618e1d.>[\S\s]+?</td>''', re.X|re.S)
Schedules_brief_re = re.compile(r'''<div\s*class=.t_detail_trip_text[^>]+>\s*([\S\s]+?)\s*</div>\s*<div''', re.X|re.S)
Schedules_eat_tra_hot_re = re.compile(r'''<div\s*class=.t_detail_trip_text2[^>]+>\s*([\S\s]+?)\s*</div>''', re.X|re.S)

def getSchedules(html):
     
     schedules_place_list = Schedules_place_re.findall(html)
     if schedules_place_list:
         #print schedules_place_list[0]
         schedules_place_list = re.compile(r'''<p>([\S\s]+?)</p>''', re.X|re.S).findall(schedules_place_list[0])
         if schedules_place_list:
             schedules = range(0,len(schedules_place_list))
             n = 0
             for plane_info in schedules_place_list:
                plane_info = strip_tags(plane_info)
                #print plane_info
                schedule = Schedule()
                if plane_info.find("第")>-1:
                    #schedule.s_place= plane_info[plane_info.find("天"):len(plane_info)]
                    #print plane_info[plane_info.find("天")+1:len(plane_info)]
                    if plane_info:
                        schedule.s_brief = plane_info.split("天")[1].replace("：","")
                        #print plane_info.split("天")[1].replace("：","")
                    if plane_info.find("住宿")>-1:
                        schedule.s_hotel = plane_info.split("住宿")[1]
                    n+=1
                    schedule.s_num = n
                    #print "serialnumber:"+str(n)
                else:
                    return "no"
                schedules[n-1]=schedule
             return schedules
     return "no"

def getPlane(html):
     plane_str = ""
     #判断是否含有线路行程天数信息
     if Schedules_place_re.findall(html):
         place_list = Schedules_place_re.findall(html)[0]
         if place_list.find('第')>-1:
             plane_info_list = route_planes_re.findall(html)
             
             n = 0
             for plane_info in plane_info_list:
                n+=1
                if n%2!=0:
                    plane_str = plane_str + "#第" + str(n/2+1) + "天# "
                    plane_str = plane_str + plane_info.strip()
                else:
                    plane_str = plane_str + "".join(plane_info.split())
         return plane_str
     return "..."
def getTrafficType(t):
    t1='-'
    if t.find('汽')>-1:
        t1='汽车'
    if t.find('动')>-1:
        t1='动车'
    if t.find('卧')>-1:
        t1='卧铺'
    if t.find('火车')>-1:
        t1='火车'
    if t.find('船')>-1:
        t1='邮轮'
    if t.find('邮')>-1:
        t1='邮轮'
    if t.find('飞')>-1:
        t1='飞机'
    if t.find('机')>-1:
        t1='飞机'
    return t1

#去掉文本标签
def strip_tags(html):
    from HTMLParser import HTMLParser
    html=html.strip()
    html=html.strip("\n")
    result=[]
    parse=HTMLParser()
    parse.handle_data=result.append
    parse.feed(html)
    parse.close()
    return "".join(result)

if __name__ == '__main__':
    script_path = os.path.dirname(os.path.realpath(__file__))
    os.chdir(script_path)
    do_debug = False

    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:], 'hdno:p:', ['help', 'debug', 'no-cache', 'only-cache', 'provider'])
    except getopt.GetoptError:
        usage()
        sys.exit()

    for opt, arg in opts:
        if opt in ('-h', '--help'):
            usage()
            sys.exit()
        elif opt in ('-d', '--debug'):
            do_debug = True
        elif opt in ('-n', '--no-cache'):
            use_cache = False
        elif opt in ('-o', '--only-cache'):
            if arg.lower() in ('no', 'n', '0'):
                only_cache = False
            else:
                only_cache = True
        elif opt in ('-p', '--provider'):
            pass

    if do_debug:
        import pdb
        pdb.set_trace()
    do_crawl()